#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :spider_ZaoJv.py
# @Time      :2024/11/12 
# @Author    :CL
# @email     :1037654919@qq.com
import re
import time

import pandas as pd
import requests
from bs4 import BeautifulSoup

from pypinyin import pinyin, Style
from util import mongo_manager
from retrying import retry
zaojv_data = mongo_manager('zaojv_data',db = 'cl_data')

def convert_pinyin_to_numeric(pinyin_string):
    # 使用pypinyin的pinyin方法，style为TONE3表示返回带数字的拼音
    result = pinyin(pinyin_string, style=Style.TONE3)
    return ''.join([item[0] for item in result])





def convert_pinyin_to_numeric(pinyin):

    # 声调字母与无声调字母的映射关系
    tone_to_no_tone_map = {
        'ā': 'a', 'á': 'a', 'ǎ': 'a', 'à': 'a',
        'ō': 'o', 'ó': 'o', 'ǒ': 'o', 'ò': 'o',
        'ē': 'e', 'é': 'e', 'ě': 'e', 'è': 'e',
        'ī': 'i', 'í': 'i', 'ǐ': 'i', 'ì': 'i',
        'ū': 'u', 'ú': 'u', 'ǔ': 'u', 'ù': 'u',
        'ǖ': 'u:', 'ǘ': 'u:', 'ǚ': 'u:', 'ǜ': 'u:'
    }

    # 声调字母与数字的映射关系
    tone_map = {
        'ā': '1', 'á': '2', 'ǎ': '3', 'à': '4',
        'ō': '1', 'ó': '2', 'ǒ': '3', 'ò': '4',
        'ē': '1', 'é': '2', 'ě': '3', 'è': '4',
        'ī': '1', 'í': '2', 'ǐ': '3', 'ì': '4',
        'ū': '1', 'ú': '2', 'ǔ': '3', 'ù': '4',
        'ǖ': '1', 'ǘ': '2', 'ǚ': '3', 'ǜ': '4'
    }
    flag_qing = True
    for char in pinyin:
        if char in tone_map:
            pinyin = pinyin.replace(char, tone_to_no_tone_map[char] ) + tone_map[char]
            flag_qing = False
            break
    if flag_qing:
        pinyin = pinyin + '5'

    return pinyin


#读取多音字
def duquduoyinzi():
    import pandas as pd
    # 读取 Excel 文件
    file_path = '多音字.xlsx'
    df = pd.read_excel(file_path,header=None)

    # 初始化字音字典
    zi_yin = {}

    # 遍历 DataFrame，将每一行的数据按字和音保存到字典
    for index, row in df.iterrows():
        zi = row[0]  # 假设第一列是字
        yin = row[1]  # 假设第二列是音

        if zi not in zi_yin:
            zi_yin[zi] = []

        # 如果音不在该字的音列表中，就添加进去
        if yin not in zi_yin[zi]:
            zi_yin[zi].append(yin)

    # 打印字音字典
    # print(zi_yin)
    return zi_yin


class ZaoJv():
    def __init__(self):
        self.headers = {
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Cache-Control": "no-cache",
            "Connection": "keep-alive",
            "Content-Type": "application/x-www-form-urlencoded",
            "Origin": "https://zaojv.com",
            "Pragma": "no-cache",
            "Referer": "https://zaojv.com/9695874.html",
            "Sec-Fetch-Dest": "document",
            "Sec-Fetch-Mode": "navigate",
            "Sec-Fetch-Site": "same-origin",
            "Sec-Fetch-User": "?1",
            "Upgrade-Insecure-Requests": "1",
            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
            "sec-ch-ua": "\"Chromium\";v=\"130\", \"Google Chrome\";v=\"130\", \"Not?A_Brand\";v=\"99\"",
            "sec-ch-ua-mobile": "?0",
            "sec-ch-ua-platform": "\"Linux\""
        }
        self.cookies = {
            "Hm_lvt_5269e069c39f6be04160a58a5db48db4": "1731411651",
            "HMACCOUNT": "8A7B5B515FA69801",
            "_ga": "GA1.2.1930766059.1731411651",
            "_gid": "GA1.2.1491252770.1731411651",
            "directGo": "1",
            "vagueSearch": "0",
            "notShowImg": "1",
            "__gads": "ID=2f91f8dd3f6549ab:T=1731411651:RT=1731411972:S=ALNI_MZXL499imVqjfWASshpoxFJTf4jEw",
            "__gpi": "UID=00000f938e81b9d8:T=1731411651:RT=1731411972:S=ALNI_MZ2cpTUWy_A7v5oUNbvlIwHxnnqfw",
            "__eoi": "ID=9890d605f5da5d17:T=1731411651:RT=1731411972:S=AA-Afjasb-eiTTMcm3B7rEVLysGv",
            "Hm_lpvt_5269e069c39f6be04160a58a5db48db4": "1731412336",
            "_ga_MCDGK9WLFC": "GS1.2.1731411651.1.1.1731412336.0.0.0"
        }
        pass
    # 输入汉字 输出组词
    @retry(wait_fixed=5, stop_max_attempt_number=3)
    def get_zuci(self,hanzi='了'):

        url = "https://zaojv.com/wordQueryDo.php"
        data = {
            "nsid": "0",
            "s": "4595742426291063331",
            "q": "",
            "wo": hanzi,
            "directGo": "1"
        }
        response = requests.post(url, headers=self.headers, cookies=self.cookies, data=data)

        # print(response.text)
        print(response.url ,response.status_code)
        if response.status_code == 200:
            return response.text



    # 输入词语链接，输出造句和拼音，信息 一目了然
    @retry(wait_fixed=5,stop_max_attempt_number=3)
    def get_sentence_pinyin(self,seed =None):
        if seed is None:
            seed ={'word': '露', 'ciyu': '露水姻缘', 'link': 'https://zaojv.com/2991416.html'}
        # url = "https://zaojv.com/5178119.html"
        url = seed['link']
        response = requests.get(url, headers=self.headers, cookies=self.cookies)

        # print(response.text)
        print(response.url,response)
        if response.status_code == 200:
            return response.text

    # 输入汉字 输
zaojv = ZaoJv()

def main1():
    duoyinzi = duquduoyinzi()
    print(duoyinzi)
    flag = 0
    for zi,yin in duoyinzi.items():
        if zi =='刺':
            flag += 1
        if flag == 0:
            continue

        time.sleep(10)
        yinpin_count ={item: 0 for item in yin}
        print(zi,yin)
        zucidata = zaojv.get_zuci(hanzi= zi)
        soup = BeautifulSoup(zucidata, 'html.parser')
        uls = soup.find_all('ul', class_='c1 ico2')
        for  ul in uls:
            datas = ul.find_all('li')
            for data in datas[:]:
                # 如果yinpin_count 里边的每个键值都大于等于20，break
                if all(yinpin_count[yp] >= 20 for yp in yinpin_count):
                    break
                # print(data.a.text, 'https://zaojv.com' + data.a['href'] )
                seed ={'word':zi,'ciyu':data.a.text[:-2],'link':'https://zaojv.com' + data.a['href']}
                print(seed)
                sentence_pinyin = zaojv.get_sentence_pinyin(seed=seed)
                if sentence_pinyin is None:
                    continue
                soup = BeautifulSoup(sentence_pinyin, 'html.parser')
                datas = soup.find('div', id='content')
                pinyin_str = datas.find('div', style='line-height:20px;float:none;margin-top:5px;margin-bottom:5px;padding:2px;background-color:#cccccc;border:1px solid #DCDDDD;width:450px;max-height:105px;overflow:auto;font-size:13px;').text
                # print(pinyin_str)
                # 检查 pinyin 字符串是否以【开头
                if pinyin_str.strip().startswith('【'):
                    # 使用正则表达式提取【】之间的内容
                    match = re.search(r'【(.*?)】', pinyin_str)
                    if match:
                        # print(match.group(1))
                        if len(match.group(1).split(' '))<3:
                            continue
                        else:
                            # print(match.group(1).split(' '))  # 打印括号之间的内容
                            index = match.group(1).split(' ')[0].find(zi)
                            yinpin_tone = match.group(1).split(' ')[index+1]
                            yinpin_num = convert_pinyin_to_numeric(yinpin_tone)
                            print(yinpin_tone,yinpin_num)
                            if yinpin_num in yinpin_count:
                                yinpin_count[yinpin_num] += 1  # 如果拼音已经出现过，则计数加1
                                if yinpin_count[yinpin_num]>20:
                                    continue

                                div_list = datas.find('div', id='all').findAll('div')
                                # 判断list长度，若小于2则取所有元素，否则取前两个
                                if len(div_list) < 2:
                                    sentences = div_list  # 如果长度小于2，取所有
                                else:
                                    sentences = div_list[:2]  # 否则取前两个
                                for num,sentence in enumerate(sentences):
                                    print(sentence.text)
                                    print(data.a.text[:-2])
                                    #设置每一条数据的格 式
                                    if data.a.text[:-2] in sentence.text:
                                        reslut = {'_id':'https://zaojv.com' + data.a['href'] +'?'+str(num),'word':zi,'yin':yinpin_num,'ciyu':data.a.text[:-2],'link':'https://zaojv.com' + data.a['href'],'sentence':sentence.text}
                                        print(reslut)
                                        try:
                                            zaojv_data.insertOne(reslut)
                                        except Exception as e:
                                            print(e)
                            else:
                                continue

                    else:
                        print("未找到【】中的内容")

        # break

# 采集预测是badcase的 数据
def main2():

    # 1. 读取 zaojv_data_merged.csv 文件
    merged_df = pd.read_csv('zaojv_data_merged.csv')

    # 2. 根据第二列去重，假设第二列是 'column1'
    # 使用 drop_duplicates 保留 'column1' 列的唯一值
    unique_df = merged_df.drop_duplicates(subset='ciyu')

    # 3. 遍历去重后的 DataFrame，每一行作为字典赋值给 seed
    seed_list = []  # 用来保存所有的字典
    for index, row in unique_df.iterrows():
        # 将每一行转换为字典,去掉最后一列
        seed = row[:-1].to_dict()
        seed_list.append(seed)

    # # 输出前几个字典示例
    # for seed in seed_list[:5]:
    #     print(seed)
    # 读取文件：zaojv_data_merged.csv
    for seed in seed_list:
        print(seed)
        page = 0
        while True:
            page +=1
            #替换页码
            seed['link'] = seed['link'].replace('.html','').split('_')[0] + '_{}.html'.format(page)
            print(seed['link'])
            sentence_pinyin = zaojv.get_sentence_pinyin(seed=seed)
            if sentence_pinyin is None:
                break
            soup = BeautifulSoup(sentence_pinyin, 'html.parser')
            datas = soup.find('div', id='content')
            sentences = datas.find('div', id='all').findAll('div')

            for num,sentence in enumerate(sentences):
                print(sentence.text)
                #设置每一条数据的格 式
                if seed['ciyu'] in sentence.text:
                    reslut = seed
                    reslut['_id'] = seed['link']+'?'+str(num)
                    reslut['sentence'] = sentence.text
                    reslut['flag'] = 'badcase'   # 设置flag为badcase,表示预测错误
                    # print(reslut)
                    try:
                        zaojv_data.insertOne(reslut)
                    except Exception as e:
                        print(e,'已经有数据了')
                        zaojv_data.updateOne({'_id':reslut['_id']},reslut)
            # 如果len(sentences)<30,说明没有数据了
            if len(sentences) < 30 or page >= 10:
                break





if __name__ == '__main__':
    print()
    main2()


#输入一个字的读音，输出读音的数字表示，比如 'liǎo' 输出 'liao3'
